from dataoutput import DataOutput
from htmldownload import HtmlDownload
from htmlparser import HtmlParser
from urlmanager import UrlManager

class SpiderMain:
    def __init__(self):
        self.manager = UrlManager()
        self.parser = HtmlParser()
        self.download = HtmlDownload()
        self.output = DataOutput()

    def working(self,url):
        self.manager.add_new_url(url)
        while self.manager.has_new_url() and self.manager.old_url_size()<100:
            try:
                new_url = self.manager.get_new_url()
                html = self.download.download(new_url)
                new_urls,data = self.parser.parser(url,html)
                self.manager.add_new_urls(new_urls)
                self.output.store_data(data)
                print('已经抓取%s个链接'%self.manager.old_url_size())

            except Exception:
                print('执行失败')
        self.output.output_html()

if __name__ == '__main__':
    spider = SpiderMain()
    spider.working('https://baike.baidu.com/item/FOAF')

